﻿using System;
using System.Collections.Generic;
using System.IO;
using ARD.Common;
using ARD.Configuration;
using ARD.EventRouting;
using ARD.ServiceManagement;
using System.Linq;
using SharpObjects.SearchEngine.Common;
using SharpObjects.SearchEngine.Common.ApplicationEvents;
using SharpObjects.SearchEngine.Common.Indexing;
using SharpObjects.SearchEngine.Common.Search;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Store;
using Lucene.Net.Search;
using SharpObjects.SearchEngine.HtmlParsing.Parser;
using System.Text;

namespace SharpObjects.SearchEngine.Indexing
{
    public class IndexingService : IConfigurationObject, IService, IIndexingService
    {
        //protected const string uri_field = "uri";
        //protected const string date_indexed_field = "__date_indexed";
        //protected const string date_modified_field = "__date_modified";

        protected EventBus eventBus = null;
        protected Logging logging = null;
        protected Lucene.Net.Analysis.Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT);
        protected int maxWordCountPerFragment;
        protected double termGroupThresholdFactor;

        public List<IndexDirectoryDescriptor> IndexDirectories { get; set; }

        public IndexingService()
        {
            this.IndexDirectories = new List<IndexDirectoryDescriptor>();
        }

        // this method returns the index directory path
        protected string GetIndexDirectoryPath(string indexDirectoryId)
        {
            var dirPath = (from dir in this.IndexDirectories
                           where dir.Id == indexDirectoryId
                           select dir.Path).Single();
            return dirPath;
        }

        // This is an event handler for the indexing application event.
        // It is called from the event bus when the indexing application event is published.
        protected void IndexingEventHandler(IApplicationEvent ev)
        {
            lock (this)
            {
                if (ev is IndexingApplicationEvent)
                {
                    IndexingApplicationEvent indEv = (IndexingApplicationEvent)ev;

                    Location location = new Location("IndexingService.IndexingEventHandler()");

                    Lucene.Net.Store.Directory indexDir = null;
                    IndexWriter writer = null;

                    try
                    {
                        string indexDirectory = GetIndexDirectoryPath(indEv.IndexDirectoryId);
                        DirectoryInfo di = new DirectoryInfo(indexDirectory);

                        if (!di.Exists)
                        {
                            di.Create();
                        }

                        indexDir = FSDirectory.Open(di);

                        if (IndexReader.IndexExists(indexDir))
                        {
                            writer = new IndexWriter(indexDir, this.analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED);
                        }
                        else
                        {
                            writer = new IndexWriter(indexDir, this.analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
                        }

                        if (indEv.IndexingAction == IndexingApplicationEvent.eIndexingAction.Index)
                        {
                            if (IsNewer(indexDir, indEv))
                            {
                                this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.DebugMessage(location) { Message = string.Format("Indexing '{0}'", indEv.Document.Uri) }));

                                Document indexDoc = new Document();

                                indexDoc.Add(new Field(DefaultIndexFields.uri_field, indEv.Document.Uri, Field.Store.YES, Field.Index.NOT_ANALYZED));

                                string dateIndexed = DateTime.Now.ToString("s");
                                indexDoc.Add(new Field(DefaultIndexFields.date_indexed_field, dateIndexed, Field.Store.YES, Field.Index.NOT_ANALYZED));

                                string dateModified = indEv.Document.DateModified.ToString("s");
                                indexDoc.Add(new Field(DefaultIndexFields.date_modified_field, dateModified, Field.Store.YES, Field.Index.NOT_ANALYZED));

                                if (indEv.Document != null)
                                {
                                    AddDocumentFields(indEv, indexDoc);

                                    writer.DeleteDocuments(new TermQuery(new Term(DefaultIndexFields.uri_field, indEv.Document.Uri)));
                                    writer.AddDocument(indexDoc);
                                    writer.Commit();

                                    // building term vectors and term groups. It has to be done after the document has been indexed
                                    // at least once, because we are using statistics, in the index

                                    AddTermVectorsAndTermGroups(indEv, indexDoc);

                                    writer.DeleteDocuments(new TermQuery(new Term(DefaultIndexFields.uri_field, indEv.Document.Uri)));
                                    writer.AddDocument(indexDoc);

                                    writer.Optimize();
                                }
                            }
                            else
                            {
                                this.eventBus.Publish(typeof (LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.DebugMessage(location) {Message = string.Format("Document is up to date: '{0}'", indEv.Document.Uri)}));
                            }
                        }
                        else if(indEv.IndexingAction == IndexingApplicationEvent.eIndexingAction.Delete)
                        {
                            this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.DebugMessage(location) { Message = string.Format("Deleting '{0}'", indEv.Document.Uri) }));
                            DeleteDocument(indexDir, indEv.Document.Uri);
                        }
                        else
                        {
                            this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent(new LoggingApplicationEvent.DebugMessage(location) { Message = string.Format("Unknown indexing action: '{0}'", indEv.IndexingAction) }));
                        }
                    }
                    catch (Exception ex)
                    {
                        this.eventBus.Publish(typeof(LoggingApplicationEvent), new LoggingApplicationEvent() { Message = new LoggingApplicationEvent.ErrorMessage(location, ex) });
                    }
                    finally
                    {
                        if (indexDir != null && writer == null)
                        {
                            indexDir.Close();
                        }
                        else if (writer != null)
                        {
                            writer.Close();
                        }
                    }
                }
            }
        }

        private void AddTermVectorsAndTermGroups(IndexingApplicationEvent indEv, Document indexDoc)
        {
            for (int k = 0; k < indEv.Document.Fields.Count; k++)
            {
                IndexField indField = indEv.Document.Fields[k];

                if (indField.BuildTermVector) //can't build term groups without term vector
                {
                    string termVectorFieldName = indField.Name + "_term_vector";

                    GetCharacterizerApplicationEvent getCharacterizerApplicationEvent =
                        new GetCharacterizerApplicationEvent()
                        {
                            Uri = indEv.Document.Uri,
                            Field = indField.Name,
                            IndexDirectoryId = indEv.IndexDirectoryId,
                            DesirableTermCount = int.MaxValue,
                            MaxTermCount = int.MaxValue,
                            ThresholdFactor = 0.0
                        };

                    this.eventBus.Publish(getCharacterizerApplicationEvent.GetType(), getCharacterizerApplicationEvent);
                    Dictionary<string, double> fullCharacterizer = getCharacterizerApplicationEvent.Characterizer;

                    getCharacterizerApplicationEvent = new GetCharacterizerApplicationEvent()
                    {
                        Uri = indEv.Document.Uri,
                        Field = indField.Name
                    };
                    this.eventBus.Publish(getCharacterizerApplicationEvent.GetType(), getCharacterizerApplicationEvent);

                    Dictionary<string, double> characterizer = getCharacterizerApplicationEvent.Characterizer;

                    foreach (string term in characterizer.Keys)
                    {
                        string item = term + ":" + characterizer[term].ToString("0.0000");
                        Field termVectorField = new Field(termVectorFieldName, item, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO);
                        indexDoc.Add(termVectorField);
                    }

                    if (indField.BuildTermGroups)
                    {
                        string termGroupsFieldname = indField.Name + "_term_groups";

                        TermGroupExtractor termGroupExtractor = new TermGroupExtractor(this.termGroupThresholdFactor, this.maxWordCountPerFragment);
                        List<List<TermVectorItem>> groups = termGroupExtractor.ProcessField(indField, fullCharacterizer);

                        foreach (List<TermVectorItem> termVectorItems in groups)
                        {
                            StringBuilder groupBuilder = new StringBuilder();

                            foreach (TermVectorItem item in termVectorItems)
                            {
                                groupBuilder.Append(item.Term + ":" + item.Score.ToString("0.0000") + ";");
                            }

                            string group = groupBuilder.ToString();
                            Field termGroupField = new Field(termGroupsFieldname, group, Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO);
                            indexDoc.Add(termGroupField);
                        }
                    }
                }
            }
            
        }

        private void AddDocumentFields(IndexingApplicationEvent indEv, Document indexDoc)
        {
            for (int k = 0; k < indEv.Document.Fields.Count; k++)
            {
                IndexField indField = indEv.Document.Fields[k];

                Field.Store store = indField.StoreInIndex ? Field.Store.YES : Field.Store.NO;
                Field.Index index = indField.Analyze ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED;
                Field.TermVector termVector = indField.BuildTermVector ? Field.TermVector.YES : Field.TermVector.NO;

                string text;
                string title;

                if (indField.IsHtml)
                {
                    ParseHtml(indField.Text, out title, out text);
                }
                else
                {
                    title = null;
                    text = indField.Text;
                }

                if (title != null)
                {
                    Field titleField = new Field("title", title, store, index, termVector);
                    titleField.SetBoost((float)2.0);
                    indexDoc.Add(titleField);
                }

                Field field = new Field(indField.Name, text, store, index, termVector);
                indexDoc.Add(field);
            }            
        }

        private List<string> BuildTermCombinations(IndexField indexField)
        {
            List<string> combinations = new List<string>();



            return combinations;
        }

        private void ParseHtml(string html, out string title, out string text)
        {
            title = null;
            text = null;

            HtmlParser htmlParser = new HtmlParser(html);
            HtmlParserDocument doc = htmlParser.Parse();
            StringBuilder sb = new StringBuilder();

            if(doc.Title != null)
            {
                title = doc.Title.Text;
                sb.Append(doc.Title.Text);
                sb.Append(" ");
            }

            foreach(TextFragment textFragment in doc.TextForIndexing)
            {
                sb.Append(textFragment.Text);
                sb.Append(" ");
            }

            text = sb.ToString();
        }

        protected bool IsNewer(Lucene.Net.Store.Directory dir, IndexingApplicationEvent indEv)
        {
            if(indEv.IndexIfNotNewer)
            {
                return true;
            }

            ISourceDocument doc = indEv.Document;

            lock (this)
            {
                if (doc.DateModified > DateTime.MinValue)
                {
                    // clear milliseconds
                    string docDateModifiedString = doc.DateModified.ToString("s");
                    DateTime docDateModified = Convert.ToDateTime(docDateModifiedString);

                    IndexSearcher searcher = new IndexSearcher(dir, true);
                    Query query = new TermQuery(new Term(DefaultIndexFields.uri_field, doc.Uri));
                    TopDocs topDocs = searcher.Search(query, 1);

                    if(topDocs.scoreDocs.Length > 0)
                    {
                        Document resultDoc = searcher.Doc(topDocs.scoreDocs[0].doc);
                        Field field = resultDoc.GetField(DefaultIndexFields.date_modified_field);
                        DateTime dateModified = Convert.ToDateTime(field.StringValue());

                        if (docDateModified > dateModified)
                        {
                            return true;
                        }
                        else
                        {
                            return false;
                        }
                    }
                    else
                    {
                        return true;
                    }
                }

                return false;
            }
        }

        protected void DeleteDocument(IndexWriter writer, string uri)
        {
            lock (this)
            {
                writer.DeleteDocuments(new TermQuery(new Term(DefaultIndexFields.uri_field, uri)));
            }
        }

        protected void DeleteDocument(Lucene.Net.Store.Directory dir, string uri)
        {
            lock (this)
            {
                IndexReader reader = IndexReader.Open(dir, false);
                reader.DeleteDocuments(new Term(DefaultIndexFields.uri_field, uri));
                reader.Close();
            }
        }

        #region IConfigurationObject Members

        public string Id { get; set; }

        public void Initialize(System.Xml.XmlElement initialization_section)
        {
            XmlConfig init = new XmlConfig(initialization_section);
            this.Id = init.GetString("@id", this.Id);

            this.IndexDirectories = init.GetObjects<IndexDirectoryDescriptor>("index_directory", IndexDirectoryDescriptor.Assembly, IndexDirectoryDescriptor.Class);

            for(int k = 0; k < this.IndexDirectories.Count; k++)
            {
                IndexDirectoryDescriptor indexDir = this.IndexDirectories[k];

                if (!Path.IsPathRooted(indexDir.Path))
                {
                    DirectoryInfo di = new DirectoryInfo(indexDir.Path);
                    indexDir.Path = di.FullName;
                }
            }

            this.eventBus = ServiceManager.Instance.GetService<EventBus>("default_event_bus");
            this.logging = new Logging(this.eventBus);
            this.eventBus.Subscribe(typeof(IndexingApplicationEvent), new EventBus.EventHandler(IndexingEventHandler));

            this.maxWordCountPerFragment = init.GetInt("max_word_count_per_fragment", 50);
            this.termGroupThresholdFactor = init.GetDouble("term_group_threshold_factor", 0.5);
         }

        #endregion

        #region IService Members

        public string ServiceId
        {
            get { return this.Id; }
        }

        public ServiceManager ServiceManager { get; set; }

        public Type ServiceType
        {
            get { return typeof(IService); }
        }

        #endregion

    }
}
